Finish plugin support
authorAlex Crichton <alex@alexcrichton.com>
Fri, 11 Jul 2014 18:22:07 +0000 (11:22 -0700)
committerAlex Crichton <alex@alexcrichton.com>
Fri, 11 Jul 2014 19:20:25 +0000 (12:20 -0700)
This commit implements full support for plugins by answering the question of
whether any target needed as a plugin or needed as a target dependency. This
commit builds on the previous abstractions to enable parallel compilation
wherever possible.

src/cargo/ops/cargo_rustc/context.rs
src/cargo/ops/cargo_rustc/fingerprint.rs
src/cargo/ops/cargo_rustc/mod.rs
tests/test_cargo_compile.rs
tests/test_cargo_cross_compile.rs

index b8b0cee0f22a6c5465da65433a8b5f070fc08573..56a5ab63bfea724ab3bcbb845e05f3ba6ac26d1f 100644 (file)
@@ -1,28 +1,41 @@
 use std::io::IoError;
 use std::io;
 use std::str;
+use std::collections::{HashMap, HashSet};
 
-use core::{Package, PackageSet, Resolve, Target};
+use core::{Package, PackageId, PackageSet, Resolve, Target};
 use util;
 use util::{CargoResult, ChainError, internal, Config};
 
+#[deriving(Show)]
+pub enum PlatformRequirement {
+    Target,
+    Plugin,
+    PluginAndTarget,
+}
+
 pub struct Context<'a, 'b> {
-    pub deps_dir: Path,
     pub primary: bool,
     pub rustc_version: String,
     pub config: &'b mut Config<'b>,
 
     dest: Path,
+    host_dest: Path,
+    deps_dir: Path,
+    host_deps_dir: Path,
     host_dylib: (String, String),
     package_set: &'a PackageSet,
     resolve: &'a Resolve,
     target_dylib: (String, String),
+    requirements: HashMap<(&'a PackageId, &'a str), PlatformRequirement>,
 }
 
 impl<'a, 'b> Context<'a, 'b> {
     pub fn new(resolve: &'a Resolve, deps: &'a PackageSet,
                config: &'b mut Config<'b>,
-               dest: Path, deps_dir: Path) -> CargoResult<Context<'a, 'b>> {
+               dest: Path, deps_dir: Path,
+               host_dest: Path, host_deps_dir: Path)
+               -> CargoResult<Context<'a, 'b>> {
         let target_dylib = try!(Context::dylib_parts(config.target()));
         let host_dylib = if config.target().is_none() {
             target_dylib.clone()
@@ -39,6 +52,9 @@ impl<'a, 'b> Context<'a, 'b> {
             config: config,
             target_dylib: target_dylib,
             host_dylib: host_dylib,
+            requirements: HashMap::new(),
+            host_dest: host_dest,
+            host_deps_dir: host_deps_dir,
         })
     }
 
@@ -72,17 +88,29 @@ impl<'a, 'b> Context<'a, 'b> {
 
     /// Prepare this context, ensuring that all filesystem directories are in
     /// place.
-    pub fn prepare(&self, pkg: &Package) -> CargoResult<()> {
+    pub fn prepare(&mut self, pkg: &'a Package) -> CargoResult<()> {
         debug!("creating target dir; path={}", self.dest.display());
 
         try!(self.mk_target(&self.dest).chain_error(||
             internal(format!("Couldn't create the target directory for {} at {}",
                      pkg.get_name(), self.dest.display()))));
+        try!(self.mk_target(&self.host_dest).chain_error(||
+            internal(format!("Couldn't create the host directory for {} at {}",
+                     pkg.get_name(), self.dest.display()))));
 
         try!(self.mk_target(&self.deps_dir).chain_error(||
             internal(format!("Couldn't create the directory for dependencies for {} at {}",
                      pkg.get_name(), self.deps_dir.display()))));
 
+        try!(self.mk_target(&self.host_deps_dir).chain_error(||
+            internal(format!("Couldn't create the directory for dependencies for {} at {}",
+                     pkg.get_name(), self.deps_dir.display()))));
+
+        let targets = pkg.get_targets().iter();
+        for target in targets.filter(|t| t.get_profile().is_compile()) {
+            self.build_requirements(pkg, target, Target, &mut HashSet::new());
+        }
+
         Ok(())
     }
 
@@ -90,6 +118,30 @@ impl<'a, 'b> Context<'a, 'b> {
         io::fs::mkdir_recursive(target, io::UserRWX)
     }
 
+    fn build_requirements(&mut self, pkg: &'a Package, target: &'a Target,
+                          req: PlatformRequirement,
+                          visiting: &mut HashSet<&'a PackageId>) {
+        if !visiting.insert(pkg.get_package_id()) { return }
+
+        let key = (pkg.get_package_id(), target.get_name());
+        let req = if target.get_profile().is_plugin() {Plugin} else {req};
+        self.requirements.insert_or_update_with(key, req, |_, v| {
+            *v = v.combine(req);
+        });
+
+        for &(pkg, dep) in self.dep_targets(pkg).iter() {
+            self.build_requirements(pkg, dep, req, visiting);
+        }
+
+        visiting.remove(&pkg.get_package_id());
+    }
+
+    pub fn get_requirement(&self, pkg: &'a Package,
+                           target: &'a Target) -> PlatformRequirement {
+        self.requirements.find(&(pkg.get_package_id(), target.get_name()))
+            .map(|a| *a).unwrap_or(Target)
+    }
+
     /// Switch this context over to being the primary compilation unit,
     /// affecting the output of `dest()` and such.
     pub fn primary(&mut self) {
@@ -97,8 +149,17 @@ impl<'a, 'b> Context<'a, 'b> {
     }
 
     /// Return the destination directory for output.
-    pub fn dest<'a>(&'a self) -> &'a Path {
-        if self.primary {&self.dest} else {&self.deps_dir}
+    pub fn dest<'a>(&'a self, plugin: bool) -> &'a Path {
+        if self.primary {
+            if plugin {&self.host_dest} else {&self.dest}
+        } else {
+            self.deps_dir(plugin)
+        }
+    }
+
+    /// Return the destination directory for dependencies.
+    pub fn deps_dir<'a>(&'a self, plugin: bool) -> &'a Path {
+        if plugin {&self.host_deps_dir} else {&self.deps_dir}
     }
 
     /// Return the (prefix, suffix) pair for dynamic libraries.
@@ -126,7 +187,7 @@ impl<'a, 'b> Context<'a, 'b> {
 
     /// For a package, return all targets which are registered as dependencies
     /// for that package.
-    pub fn dep_targets(&self, pkg: &Package) -> Vec<Target> {
+    pub fn dep_targets(&self, pkg: &Package) -> Vec<(&'a Package, &'a Target)> {
         let deps = match self.resolve.deps(pkg.get_package_id()) {
             None => return vec!(),
             Some(deps) => deps,
@@ -139,9 +200,18 @@ impl<'a, 'b> Context<'a, 'b> {
         .filter_map(|pkg| {
             pkg.get_targets().iter().find(|&t| {
                 t.is_lib() && t.get_profile().is_compile()
-            })
+            }).map(|t| (pkg, t))
         })
-        .map(|t| t.clone())
         .collect()
     }
 }
+
+impl PlatformRequirement {
+    fn combine(self, other: PlatformRequirement) -> PlatformRequirement {
+        match (self, other) {
+            (Target, Target) => Target,
+            (Plugin, Plugin) => Plugin,
+            _ => PluginAndTarget,
+        }
+    }
+}
index 7bee9f9b87b2e8394a83c719f4db3f7df766b8f7..68b121d0c7ec66ff0548ed2ef6e67252f4ad7165 100644 (file)
@@ -18,8 +18,8 @@ use super::context::Context;
 /// compilation, returning the job as the second part of the tuple.
 pub fn prepare(cx: &mut Context, pkg: &Package,
                targets: &[&Target]) -> CargoResult<(Freshness, Job)> {
-    let fingerprint_loc = cx.dest().join(format!(".{}.fingerprint",
-                                                 pkg.get_name()));
+    let fingerprint_loc = cx.dest(false).join(format!(".{}.fingerprint",
+                                                      pkg.get_name()));
 
     let (is_fresh, fingerprint) = try!(is_fresh(pkg, &fingerprint_loc,
                                                 cx, targets));
index 067acd0dd590a886b0edbe12f1e50a7cba909b93..e19daae8f1810f0f9b4f91e947d45ca0b7a9a13c 100644 (file)
@@ -5,7 +5,7 @@ use util::{Config, Freshness};
 
 use self::job::Job;
 use self::job_queue::JobQueue;
-use self::context::Context;
+use self::context::{Context, PlatformRequirement, Target, Plugin, PluginAndTarget};
 
 mod context;
 mod fingerprint;
@@ -32,7 +32,7 @@ fn uniq_target_dest<'a>(targets: &[&'a Target]) -> Option<&'a str> {
     curr.unwrap()
 }
 
-pub fn compile_targets<'a>(env: &str, targets: &[&Target], pkg: &Package,
+pub fn compile_targets<'a>(env: &str, targets: &[&'a Target], pkg: &'a Package,
                            deps: &PackageSet, resolve: &'a Resolve,
                            config: &'a mut Config<'a>) -> CargoResult<()>
 {
@@ -42,13 +42,18 @@ pub fn compile_targets<'a>(env: &str, targets: &[&Target], pkg: &Package,
 
     debug!("compile_targets; targets={}; pkg={}; deps={}", targets, pkg, deps);
 
+    let host_dir = pkg.get_absolute_target_dir()
+                        .join(uniq_target_dest(targets).unwrap_or(""));
+    let host_deps_dir = host_dir.join("deps");
+
     let target_dir = pkg.get_absolute_target_dir()
                         .join(config.target().unwrap_or(""))
                         .join(uniq_target_dest(targets).unwrap_or(""));
     let deps_target_dir = target_dir.join("deps");
 
     let mut cx = try!(Context::new(resolve, deps, config,
-                                   target_dir, deps_target_dir));
+                                   target_dir, deps_target_dir,
+                                   host_dir, host_deps_dir));
 
     // First ensure that the destination directory exists
     try!(cx.prepare(pkg));
@@ -79,8 +84,10 @@ pub fn compile_targets<'a>(env: &str, targets: &[&Target], pkg: &Package,
     JobQueue::new(cx.config, jobs).execute()
 }
 
-fn compile<'a>(targets: &[&Target], pkg: &'a Package, cx: &mut Context,
-               jobs: &mut Vec<(&'a Package, Freshness, Job)>) -> CargoResult<()> {
+fn compile<'a, 'b>(targets: &[&'a Target], pkg: &'a Package,
+                   cx: &mut Context<'a, 'b>,
+                   jobs: &mut Vec<(&'a Package, Freshness, Job)>)
+                   -> CargoResult<()> {
     debug!("compile_pkg; pkg={}; targets={}", pkg, targets);
 
     if targets.is_empty() {
@@ -104,11 +111,12 @@ fn compile<'a>(targets: &[&Target], pkg: &'a Package, cx: &mut Context,
     // interdependencies.
     let (mut libs, mut bins) = (Vec::new(), Vec::new());
     for &target in targets.iter() {
-        let job = rustc(pkg, target, cx);
+        let req = cx.get_requirement(pkg, target);
+        let jobs = rustc(pkg, target, cx, req);
         if target.is_lib() {
-            libs.push(job);
+            libs.push_all_move(jobs);
         } else {
-            bins.push(job);
+            bins.push_all_move(jobs);
         }
     }
 
@@ -134,13 +142,15 @@ fn compile<'a>(targets: &[&Target], pkg: &'a Package, cx: &mut Context,
 
 fn compile_custom(pkg: &Package, cmd: &str,
                   cx: &Context) -> Job {
-    // FIXME: this needs to be smarter about splitting
+    // TODO: this needs to be smarter about splitting
     let mut cmd = cmd.split(' ');
+    // TODO: this shouldn't explicitly pass `false` for dest/deps_dir, we may
+    //       be building a C lib for a plugin
     let mut p = util::process(cmd.next().unwrap())
                      .cwd(pkg.get_root())
-                     .env("OUT_DIR", Some(cx.dest().as_str()
+                     .env("OUT_DIR", Some(cx.dest(false).as_str()
                                             .expect("non-UTF8 dest path")))
-                     .env("DEPS_DIR", Some(cx.deps_dir.as_str()
+                     .env("DEPS_DIR", Some(cx.deps_dir(false).as_str()
                                              .expect("non-UTF8 deps path")))
                      .env("TARGET", cx.config.target());
     for arg in cmd {
@@ -152,56 +162,73 @@ fn compile_custom(pkg: &Package, cmd: &str,
     })
 }
 
-fn rustc(package: &Package, target: &Target, cx: &mut Context) -> Job {
+fn rustc(package: &Package, target: &Target,
+         cx: &mut Context, req: PlatformRequirement) -> Vec<Job> {
     let crate_types = target.rustc_crate_types();
     let root = package.get_root();
 
-    log!(5, "root={}; target={}; crate_types={}; dest={}; deps={}; verbose={}",
-         root.display(), target, crate_types, cx.dest().display(),
-         cx.deps_dir.display(), cx.primary);
+    log!(5, "root={}; target={}; crate_types={}; dest={}; deps={}; \
+             verbose={}; req={}",
+         root.display(), target, crate_types, cx.dest(false).display(),
+         cx.deps_dir(false).display(), cx.primary, req);
 
     let primary = cx.primary;
-    let rustc = prepare_rustc(package, target, crate_types, cx);
+    let rustcs = prepare_rustc(package, target, crate_types, cx, req);
 
-    log!(5, "command={}", rustc);
+    log!(5, "commands={}", rustcs);
 
     let _ = cx.config.shell().verbose(|shell| {
-        shell.status("Running", rustc.to_string())
+        for rustc in rustcs.iter() {
+            try!(shell.status("Running", rustc.to_string()));
+        }
+        Ok(())
     });
 
-    Job::new(proc() {
-        if primary {
-            log!(5, "executing primary");
-            try!(rustc.exec().map_err(|err| human(err.to_string())))
-        } else {
-            log!(5, "executing deps");
-            try!(rustc.exec_with_output().and(Ok(())).map_err(|err| {
-                human(err.to_string())
-            }))
-        }
-        Ok(Vec::new())
-    })
+    rustcs.move_iter().map(|rustc| {
+        Job::new(proc() {
+            if primary {
+                log!(5, "executing primary");
+                try!(rustc.exec().map_err(|err| human(err.to_string())))
+            } else {
+                log!(5, "executing deps");
+                try!(rustc.exec_with_output().and(Ok(())).map_err(|err| {
+                    human(err.to_string())
+                }))
+            }
+            Ok(Vec::new())
+        })
+    }).collect()
 }
 
 fn prepare_rustc(package: &Package, target: &Target, crate_types: Vec<&str>,
-                 cx: &Context) -> ProcessBuilder
-{
+                 cx: &Context, req: PlatformRequirement) -> Vec<ProcessBuilder> {
     let root = package.get_root();
-    let mut args = Vec::new();
-
-    build_base_args(&mut args, target, crate_types, cx);
-    build_deps_args(&mut args, package, cx);
-
-    util::process("rustc")
-        .cwd(root.clone())
-        .args(args.as_slice())
-        .env("RUST_LOG", None) // rustc is way too noisy
+    let mut target_args = Vec::new();
+    build_base_args(&mut target_args, target, crate_types.as_slice(), cx, false);
+    build_deps_args(&mut target_args, package, cx, false);
+
+    let mut plugin_args = Vec::new();
+    build_base_args(&mut plugin_args, target, crate_types.as_slice(), cx, true);
+    build_deps_args(&mut plugin_args, package, cx, true);
+
+    let base = util::process("rustc").cwd(root.clone());
+
+    match req {
+        Target => vec![base.args(target_args.as_slice())],
+        Plugin => vec![base.args(plugin_args.as_slice())],
+        PluginAndTarget if cx.config.target().is_none() =>
+            vec![base.args(target_args.as_slice())],
+        PluginAndTarget =>
+            vec![base.clone().args(target_args.as_slice()),
+                 base.args(plugin_args.as_slice())],
+    }
 }
 
 fn build_base_args(into: &mut Args,
                    target: &Target,
-                   crate_types: Vec<&str>,
-                   cx: &Context)
+                   crate_types: &[&str],
+                   cx: &Context,
+                   plugin: bool)
 {
     let metadata = target.get_metadata();
 
@@ -216,7 +243,6 @@ fn build_base_args(into: &mut Args,
         into.push(crate_type.to_string());
     }
 
-    let out = cx.dest().clone();
     let profile = target.get_profile();
 
     if profile.get_opt_level() != 0 {
@@ -244,16 +270,11 @@ fn build_base_args(into: &mut Args,
         None => {}
     }
 
-    if target.is_lib() {
-        into.push("--out-dir".to_string());
-        into.push(out.display().to_string());
-    } else {
-        into.push("-o".to_string());
-        into.push(out.join(target.get_name()).display().to_string());
-    }
+    into.push("--out-dir".to_string());
+    into.push(cx.dest(plugin).display().to_string());
 
     match cx.config.target() {
-        Some(target) if !profile.is_plugin() => {
+        Some(target) if !plugin => {
             into.push("--target".to_string());
             into.push(target.to_string());
         }
@@ -261,17 +282,18 @@ fn build_base_args(into: &mut Args,
     }
 }
 
-fn build_deps_args(dst: &mut Args, package: &Package, cx: &Context) {
+fn build_deps_args(dst: &mut Args, package: &Package, cx: &Context,
+                   plugin: bool) {
     dst.push("-L".to_string());
-    dst.push(cx.dest().display().to_string());
+    dst.push(cx.dest(plugin).display().to_string());
     dst.push("-L".to_string());
-    dst.push(cx.deps_dir.display().to_string());
+    dst.push(cx.deps_dir(plugin).display().to_string());
 
-    for target in cx.dep_targets(package).iter() {
+    for &(_, target) in cx.dep_targets(package).iter() {
         dst.push("--extern".to_string());
         dst.push(format!("{}={}/{}",
                  target.get_name(),
-                 cx.deps_dir.display(),
+                 cx.deps_dir(target.get_profile().is_plugin()).display(),
                  cx.target_filename(target)));
     }
 }
index 57ef09cebceddc413fd09b52ab9d9d4ee4349540..7c2d0affb11ef86a92633bb5f46ddc6b01c9e60f 100644 (file)
@@ -93,8 +93,8 @@ test!(cargo_compile_with_invalid_code {
 {filename}:1 invalid rust code!
              ^~~~~~~
 Could not execute process \
-`rustc {filename} --crate-name foo --crate-type bin -o {} -L {} -L {}` (status=101)\n",
-            target.join("foo").display(),
+`rustc {filename} --crate-name foo --crate-type bin --out-dir {} -L {} -L {}` (status=101)\n",
+            target.display(),
             target.display(),
             target.join("deps").display(),
             filename = format!("src{}foo.rs", path::SEP)).as_slice()));
index 4bc4ddc604f91ed19dbac053a39466f323ab6733..9483e82d0995d2e03e2397f0d355eeb0ddad7d84 100644 (file)
@@ -15,7 +15,7 @@ fn setup() {
 fn alternate() -> &'static str {
     match os::consts::SYSNAME {
         "linux" => "i686-unknown-linux-gnu",
-        "darwin" => "i686-apple-darwin",
+        "macos" => "i686-apple-darwin",
         _ => unreachable!(),
     }
 }
@@ -75,4 +75,158 @@ test!(simple_deps {
       execs().with_status(0));
 })
 
+test!(plugin_deps {
+    let foo = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+
+            [dependencies.bar]
+            path = "../bar"
+
+            [dependencies.baz]
+            path = "../baz"
+        "#)
+        .file("src/main.rs", r#"
+            #![feature(phase)]
+            #[phase(plugin)]
+            extern crate bar;
+            extern crate baz;
+            fn main() {
+                assert_eq!(bar!(), baz::baz());
+            }
+        "#);
+    let bar = project("bar")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "bar"
+            version = "0.0.1"
+            authors = []
 
+            [[lib]]
+            name = "bar"
+            plugin = true
+        "#)
+        .file("src/lib.rs", r#"
+            #![feature(plugin_registrar, quote)]
+
+            extern crate rustc;
+            extern crate syntax;
+
+            use rustc::plugin::Registry;
+            use syntax::ast::TokenTree;
+            use syntax::codemap::Span;
+            use syntax::ext::base::{ExtCtxt, MacExpr, MacResult};
+
+            #[plugin_registrar]
+            pub fn foo(reg: &mut Registry) {
+                reg.register_macro("bar", expand_bar);
+            }
+
+            fn expand_bar(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
+                          -> Box<MacResult> {
+                MacExpr::new(quote_expr!(cx, 1i))
+            }
+        "#);
+    let baz = project("baz")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "baz"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", "pub fn baz() -> int { 1 }");
+    bar.build();
+    baz.build();
+
+    let target = alternate();
+    assert_that(foo.cargo_process("cargo-build").arg("--target").arg(target),
+                execs().with_status(0));
+    assert_that(&foo.target_bin(target, "main"), existing_file());
+
+    assert_that(
+      process(foo.target_bin(target, "main")),
+      execs().with_status(0));
+})
+
+test!(plugin_to_the_max {
+    let foo = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+
+            [dependencies.bar]
+            path = "../bar"
+
+            [dependencies.baz]
+            path = "../baz"
+        "#)
+        .file("src/main.rs", r#"
+            #![feature(phase)]
+            #[phase(plugin)]
+            extern crate bar;
+            extern crate baz;
+            fn main() {
+                assert_eq!(bar!(), baz::baz());
+            }
+        "#);
+    let bar = project("bar")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "bar"
+            version = "0.0.1"
+            authors = []
+
+            [[lib]]
+            name = "bar"
+            plugin = true
+
+            [dependencies.baz]
+            path = "../baz"
+        "#)
+        .file("src/lib.rs", r#"
+            #![feature(plugin_registrar, quote)]
+
+            extern crate rustc;
+            extern crate syntax;
+            extern crate baz;
+
+            use rustc::plugin::Registry;
+            use syntax::ast::TokenTree;
+            use syntax::codemap::Span;
+            use syntax::ext::base::{ExtCtxt, MacExpr, MacResult};
+
+            #[plugin_registrar]
+            pub fn foo(reg: &mut Registry) {
+                reg.register_macro("bar", expand_bar);
+            }
+
+            fn expand_bar(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
+                          -> Box<MacResult> {
+                MacExpr::new(quote_expr!(cx, baz::baz()))
+            }
+        "#);
+    let baz = project("baz")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "baz"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", "pub fn baz() -> int { 1 }");
+    bar.build();
+    baz.build();
+
+    let target = alternate();
+    assert_that(foo.cargo_process("cargo-build").arg("--target").arg(target),
+                execs().with_status(0));
+    assert_that(&foo.target_bin(target, "main"), existing_file());
+
+    assert_that(
+      process(foo.target_bin(target, "main")),
+      execs().with_status(0));
+})